package com.kaowen.spider;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.handlers.ArrayListHandler;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.kaowen.spider.util.DBManager;

/**
 * 网络爬虫 Gk66
 * 
 * @author thinkpad
 */
public class SpiderForGk66 {

	private static Logger logger = Logger.getLogger(SpiderForGk66.class);
	private Gson gson = new Gson();
	private HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); // 创建HttpClientBuilder
	private Connection conn = null;

	private String cookie = null;
	private String __VIEWSTATE = null;
	private String __EVENTVALIDATION = null;

	private Map<String, String> nf = null;
	private Map<String, String> wl = null;
	private Map<String, String> bz = null;

	private String rand = null;

	public SpiderForGk66(String cookie) {
		this.cookie = cookie;
		nf = new TreeMap<String, String>();
		for (int i = 2014; i >= 2006; i--) {
			nf.put(String.valueOf(i).substring(2, 4), i + "年");
		}
		wl = gson.fromJson("{'l':'理科','w':'文科'}", (new TypeToken<Map<String, String>>() {
		}).getType());
		bz = gson.fromJson("{'b':'本科','z':'专科'}", (new TypeToken<Map<String, String>>() {
		}).getType());
		try {
			conn = DBManager.getConnection();
		} catch (SQLException e) {
			e.printStackTrace();
		}
		init();
	}

	private void init() {
		String url = "http://14wj.gk66.cn/wj/zy.aspx";
		Document document;
		try {
			document = Jsoup.connect(url).header("cookie", cookie).get();
			__VIEWSTATE = document.getElementById("__VIEWSTATE").val();
			logger.info("__VIEWSTATE:" + __VIEWSTATE);
			__EVENTVALIDATION = document.getElementById("__EVENTVALIDATION").val();
			logger.info("__EVENTVALIDATION:" + __EVENTVALIDATION);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		codewj();
		// 输入验证码
		BufferedReader strin = new BufferedReader(new InputStreamReader(System.in));
		System.out.print("请输入验证码：");
		try {
			rand = strin.readLine();
			if (validate(rand)) {
				logger.info("验证成功！");
			} else {
				logger.info("验证码错误！");
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	public void test() {
		// HttpClient
		CloseableHttpClient httpClient = httpClientBuilder.build();
		String url = "http://14wj.gk66.cn/wj/zy.aspx";
		HttpPost httpPost = new HttpPost(url);
		// 创建参数队列
		List<NameValuePair> formParams = new ArrayList<NameValuePair>();
		formParams.add(new BasicNameValuePair("__VIEWSTATE", __VIEWSTATE));
		formParams.add(new BasicNameValuePair("__EVENTVALIDATION", __EVENTVALIDATION));
		formParams.add(new BasicNameValuePair("nf", "14"));
		formParams.add(new BasicNameValuePair("wl", "l"));
		formParams.add(new BasicNameValuePair("bz", "b"));
		formParams.add(new BasicNameValuePair("zy_xf", "310"));
		formParams.add(new BasicNameValuePair("rand", "ST"));
		formParams.add(new BasicNameValuePair("ImageButton1.x", "103"));
		formParams.add(new BasicNameValuePair("ImageButton1.y", "13"));
		UrlEncodedFormEntity uefEntity;
		try {
			uefEntity = new UrlEncodedFormEntity(formParams, "UTF-8");
			httpPost.setEntity(uefEntity);
			httpPost.setHeader("cookie", cookie);
			logger.info("executing request " + httpPost.getURI());
			CloseableHttpResponse response = httpClient.execute(httpPost);
			try {
				Header[] headers = response.getAllHeaders();
				StringBuffer sb = new StringBuffer();
				for (Header header : headers) {
					sb.append(header.getName());
					sb.append(":");
					sb.append(header.getValue());
					sb.append("\n");
				}
				logger.info(sb.toString());
				;

				HttpEntity entity = response.getEntity();
				if (entity != null) {
					logger.info("Response content: " + EntityUtils.toString(entity, "UTF-8"));
				}
			} finally {
				response.close();
			}
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			// 关闭连接,释放资源
			try {
				httpClient.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}

	/**
	 * 获取验证码
	 */
	private void codewj() {
		// HttpClient
		CloseableHttpClient httpClient = httpClientBuilder.build();
		String url = "http://14wj.gk66.cn/ashx/codewj.ashx";
		HttpGet httpGet = new HttpGet(url);
		try {
			httpGet.setHeader("cookie", cookie);
			System.out.println("executing request " + httpGet.getURI());
			CloseableHttpResponse response = httpClient.execute(httpGet);
			File file = null;
			FileOutputStream fos = null;
			try {
				HttpEntity entity = response.getEntity();
				InputStream is = entity.getContent();
				file = new File("D:\\codewj.jpg");
				fos = new FileOutputStream(file);
				byte buf[] = new byte[4 * 1024];
				while ((is.read(buf)) != -1) {
					fos.write(buf);
				}
				fos.flush();
			} finally {
				response.close();
				fos.close();
			}
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			try {
				httpClient.close();
			} catch (IOException e) {
			}
		}
	}

	private List<Map<String, String>> zyajax(String nf, String wl, String bz) {
		List<Map<String, String>> rtVal = null;
		// HttpClient
		CloseableHttpClient httpClient = httpClientBuilder.build();
		String url = "http://14wj.gk66.cn/ashx/zyajax.ashx";
		HttpPost httpPost = new HttpPost(url);
		// 创建参数队列
		List<NameValuePair> formParams = new ArrayList<NameValuePair>();
		formParams.add(new BasicNameValuePair("t", "zy"));
		formParams.add(new BasicNameValuePair("nf", nf));
		formParams.add(new BasicNameValuePair("wl", wl));
		formParams.add(new BasicNameValuePair("bz", bz));
		formParams.add(new BasicNameValuePair("id", "1"));
		formParams.add(new BasicNameValuePair("pro", "liaoning"));
		UrlEncodedFormEntity uefEntity;
		try {
			uefEntity = new UrlEncodedFormEntity(formParams, "UTF-8");
			httpPost.setEntity(uefEntity);
			httpPost.setHeader("cookie", cookie);
			logger.info("executing request " + httpPost.getURI());
			CloseableHttpResponse response = httpClient.execute(httpPost);
			try {
				rtVal = gson.fromJson(EntityUtils.toString(response.getEntity(), "gb2312"),
						(new TypeToken<List<Map<String, String>>>() {
						}).getType());
			} finally {
				response.close();
			}
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			// 关闭连接,释放资源
			try {
				httpClient.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		return rtVal;
	}

	private List<Map<String, String>> pcajax(String nf, String wl, String bz, String id) {
		List<Map<String, String>> rtVal = null;
		// HttpClient
		CloseableHttpClient httpClient = httpClientBuilder.build();
		String url = "http://14wj.gk66.cn/ashx/zyajax.ashx";
		HttpPost httpPost = new HttpPost(url);
		// 创建参数队列
		List<NameValuePair> formParams = new ArrayList<NameValuePair>();
		formParams.add(new BasicNameValuePair("t", "pc"));
		formParams.add(new BasicNameValuePair("nf", nf));
		formParams.add(new BasicNameValuePair("wl", wl));
		formParams.add(new BasicNameValuePair("bz", bz));
		formParams.add(new BasicNameValuePair("id", id));
		formParams.add(new BasicNameValuePair("pro", "liaoning"));
		UrlEncodedFormEntity uefEntity;
		try {
			uefEntity = new UrlEncodedFormEntity(formParams, "UTF-8");
			httpPost.setEntity(uefEntity);
			httpPost.setHeader("cookie", cookie);
			logger.info("executing request " + httpPost.getURI());
			CloseableHttpResponse response = httpClient.execute(httpPost);
			try {
				rtVal = gson.fromJson(EntityUtils.toString(response.getEntity(), "gb2312"),
						(new TypeToken<List<Map<String, String>>>() {
						}).getType());
			} finally {
				response.close();
			}
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			// 关闭连接,释放资源
			try {
				httpClient.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		return rtVal;
	}

	private String zy(String nf, String wl, String bz, String zy_xf) {
		String rtVal = null;
		// HttpClient
		CloseableHttpClient httpClient = httpClientBuilder.build();
		String url = "http://14wj.gk66.cn/wj/zy.aspx";
		HttpPost httpPost = new HttpPost(url);
		// 创建参数队列
		List<NameValuePair> formParams = new ArrayList<NameValuePair>();
		formParams.add(new BasicNameValuePair("__VIEWSTATE", __VIEWSTATE));
		formParams.add(new BasicNameValuePair("__EVENTVALIDATION", __EVENTVALIDATION));
		formParams.add(new BasicNameValuePair("nf", nf));
		formParams.add(new BasicNameValuePair("wl", wl));
		formParams.add(new BasicNameValuePair("bz", bz));
		formParams.add(new BasicNameValuePair("zy_xf", zy_xf));
		formParams.add(new BasicNameValuePair("rand", rand));
		formParams.add(new BasicNameValuePair("ImageButton1.x", "103"));
		formParams.add(new BasicNameValuePair("ImageButton1.y", "13"));
		UrlEncodedFormEntity uefEntity;
		try {
			uefEntity = new UrlEncodedFormEntity(formParams, "UTF-8");
			httpPost.setEntity(uefEntity);
			httpPost.setHeader("cookie", cookie);
			// logger.info("executing request " + httpPost.getURI());
			CloseableHttpResponse response = httpClient.execute(httpPost);
			try {
				if (response.containsHeader("Location")) {
					rtVal = response.getFirstHeader("Location").getValue();
				}
			} finally {
				response.close();
			}
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			// 关闭连接,释放资源
			try {
				httpClient.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		return rtVal;
	}

	private boolean validate(String code) {
		boolean rtVal = false;
		// HttpClient
		CloseableHttpClient httpClient = httpClientBuilder.build();
		String url = "http://14wj.gk66.cn/ashx/validate.ashx";
		HttpPost httpPost = new HttpPost(url);
		// 创建参数队列
		List<NameValuePair> formParams = new ArrayList<NameValuePair>();
		formParams.add(new BasicNameValuePair("code", code));
		UrlEncodedFormEntity uefEntity;
		try {
			uefEntity = new UrlEncodedFormEntity(formParams, "UTF-8");
			httpPost.setEntity(uefEntity);
			httpPost.setHeader("cookie", cookie);
			logger.info("executing request " + httpPost.getURI());
			CloseableHttpResponse response = httpClient.execute(httpPost);
			try {
				HttpEntity entity = response.getEntity();
				String rt = EntityUtils.toString(entity, "gb2312");
				if ("1".equals(rt)) {
					rtVal = true;
				}
			} finally {
				response.close();
			}
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			// 关闭连接,释放资源
			try {
				httpClient.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}

		return rtVal;
	}

	public void spideZy() {
		QueryRunner queryRunner = new QueryRunner();
		for (String nf_key : nf.keySet()) {
			String nf_value = nf.get(nf_key);
			for (String wl_key : wl.keySet()) {
				String wl_value = wl.get(wl_key);
				for (String bz_key : bz.keySet()) {
					String bz_value = bz.get(bz_key);
					List<Map<String, String>> zyList = zyajax(nf_key, wl_key, bz_key);
					for (Map<String, String> zy : zyList) {
						String zy_key = zy.get("a");
						String zy_value = zy.get("b");
						String sql = "insert into SP_GK (ID,年份,年份ID,文理,文理ID,本专,本专ID,专业,专业ID) values (SP_GK_SEQ.NEXTVAL, ?,?,?,?,?,?,?,?)";
						try {
							queryRunner.update(conn, sql, nf_value, nf_key, wl_value, wl_key, bz_value, bz_key,
									zy_value, zy_key);
							String line = "\r\n" + nf_value + "," + wl_value + "," + bz_value + "," + zy_value + ","
									+ zy_key;
							System.out.print(line);
						} catch (SQLException e) {
							e.printStackTrace();
						}
					}
				}
			}
		}
	}

	public void spideUrl() {
		QueryRunner queryRunner = new QueryRunner();
		String querySql = "select t.年份id,t.文理id,t.本专id,t.专业id,t.id from sp_gk t where rownum <= 100 and t.状态 = 0";
		while (true) {
			try {
				List<Object[]> rsArray = queryRunner.query(conn, querySql, new ArrayListHandler());
				if (rsArray.size() > 0) {
					for (Object[] row : rsArray) {
						try {
							String url = zy((String) row[0], (String) row[1], (String) row[2], (String) row[3]);
							String updateSql = "update sp_gk set url = ?,状态=? where id = ?";
							queryRunner.update(conn, updateSql, "http://14wj.gk66.cn" + url, 1, row[4]);
							logger.info(row[0] + "," + row[1] + "," + row[2] + "," + row[3] + "," + row[4]);
						} catch (Exception e) {
							String updateSql = "update sp_gk set url = ?,状态=? where id = ?";
							queryRunner.update(conn, updateSql, e.toString(), 9, row[4]);
							logger.error(row[0] + "," + row[1] + "," + row[2] + "," + row[3] + "," + row[4]);
						}
					}
				} else {
					logger.info("所有数据已抓取完毕");
					break;
				}
			} catch (SQLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}

	public static void main(String[] args) {
		String cookie = "Hm_lvt_25ae7b1484a52b8a6ffdff4555060aa4=1427338168,1427787306,1427868264,1427986363; gk66_guid=GUID=717761544265213; gk66_userinfo=uid=134310009&gk_type=8&gk_name=91266740&nf=15&gk_power=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105&Pro=%e8%be%bd%e5%ae%81&sign=EAFB2DA853450F6A67671EA56AC4DA86; gk66_login=remname=91266740&rempass=6q6kYuls+kk=&remcheck=on";
		SpiderForGk66 spider = new SpiderForGk66(cookie);
		// spider.spideZy();
		spider.spideUrl();
	}
}
